import datetime
import os
from jili.core import load,save
import pandas as pd
import numpy as np
from copy import deepcopy
from tqdm import tqdm
from jili.calc.graph import graph
from jili.data.db import getdb_client
from jili.data.db import get_calender_a
from jili.tool.convert import str2datetime
def nanfill(k,arg):
    input_names=arg.get("input_names",['close'])
    fillvalue  = arg.get("fillvalue",0)
    for i in input_names:
        start = list(k[i].dropna().sort_index().index)
        if len(start) > 0:
            start = start[0]
            k.loc[start:,i]=k.loc[start:,i].fillna(fillvalue)
        else:
            k.loc[:,i] = k.loc[:,i].fillna(fillvalue)
    return k
factors={
    "ma_volume_20":{'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ma_volume_20'],
               'input': {'price': 'volume'},
               'arg': {'timeperiod': 20, 'matype': 0}, 'batch': 20},
    "ma_amount_20":{'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ma_amount_20'],
                   'input': {'price': 'amount'},
                   'arg': {'timeperiod': 20, 'matype': 0}, 'batch': 20},
    "ma_amount_10":{'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ma_amount_10'],
                   'input': {'price': 'amount'},
                   'arg': {'timeperiod': 10, 'matype': 0}, 'batch': 10},
"ma_turn_10":{'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ma_turn_10'],
                   'input': {'price': 'turn'},
                   'arg': {'timeperiod': 10, 'matype': 0}, 'batch': 10},
"ma_turn_5":{'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ma_turn_5'],
                   'input': {'price': 'turn'},
                   'arg': {'timeperiod': 5, 'matype': 0}, 'batch': 5},
    "ma_volume_10": {'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ma_volume_10'],
                     'input': {'price': 'volume'},
                     'arg': {'timeperiod': 10, 'matype': 0}, 'batch': 10},
    "ma_volume_5": {'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ma_volume_5'],
                     'input': {'price': 'volume'},
                     'arg': {'timeperiod': 5, 'matype': 0}, 'batch': 5},
"ma_close_5": {'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ma_close_5'],
                     'input': {'price': 'close'},
                     'arg': {'timeperiod': 5, 'matype': 0}, 'batch': 5},
"ma_close_10": {'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ma_close_10'],
                     'input': {'price': 'close'},
                     'arg': {'timeperiod': 10, 'matype': 0}, 'batch': 10},
"ma_close_20": {'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ma_close_20'],
                     'input': {'price': 'close'},
                     'arg': {'timeperiod': 20, 'matype': 0}, 'batch': 20},
"ma_close_40": {'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ma_close_40'],
                     'input': {'price': 'close'},
                     'arg': {'timeperiod': 40, 'matype': 0}, 'batch': 40},
"ma_close_60": {'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ma_close_60'],
                     'input': {'price': 'close'},
                     'arg': {'timeperiod': 60, 'matype': 0}, 'batch': 60},
"ma_close_120": {'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ma_close_120'],
                     'input': {'price': 'close'},
                     'arg': {'timeperiod': 120, 'matype': 0}, 'batch': 120},
"ema_close_5": {'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ema_close_5'],
                     'input': {'price': 'close'},
                     'arg': {'timeperiod': 5, 'matype': 1}, 'batch': 5},
"ema_close_10": {'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ema_close_10'],
                     'input': {'price': 'close'},
                     'arg': {'timeperiod': 10, 'matype': 1}, 'batch': 10},
"ema_close_20": {'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ema_close_20'],
                     'input': {'price': 'close'},
                     'arg': {'timeperiod': 20, 'matype': 1}, 'batch': 20},
"ema_close_40": {'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ema_close_40'],
                     'input': {'price': 'close'},
                     'arg': {'timeperiod': 40, 'matype': 1}, 'batch': 40},
"ema_close_60": {'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ema_close_60'],
                     'input': {'price': 'close'},
                     'arg': {'timeperiod': 60, 'matype': 1}, 'batch': 60},
"ema_close_120": {'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ema_close_120'],
                     'input': {'price': 'close'},
                     'arg': {'timeperiod': 120, 'matype': 1}, 'batch': 120},
}
factors1={

    "pctchg_50": {'calc_cmd': 'ta', 'cmd': 'SUM', 'out': ['pctchg_50'],
                  'input': {'price': 'pctChg'},
                  'arg': {'timeperiod': 50}, 'batch': 50},
    "pctchg_3": {'calc_cmd': 'ta', 'cmd': 'SUM', 'out': ['pctchg_3'],
                 'input': {'price': 'pctChg'},
                 'arg': {'timeperiod': 3}, 'batch': 3},
"pctchg_2":{'calc_cmd': 'ta', 'cmd': 'SUM', 'out': ['pctchg_2'],
                   'input': {'price': 'pctChg'},
                   'arg': {'timeperiod': 2}, 'batch': 2},
"pctchg_20":{'calc_cmd': 'ta', 'cmd': 'SUM', 'out': ['pctchg_20'],
                   'input': {'price': 'pctChg'},
                   'arg': {'timeperiod': 20}, 'batch': 20},
"pctchg_40":{'calc_cmd': 'ta', 'cmd': 'SUM', 'out': ['pctchg_40'],
                   'input': {'price': 'pctChg'},
                   'arg': {'timeperiod': 40}, 'batch': 40},
"pctchg_30":{'calc_cmd': 'ta', 'cmd': 'SUM', 'out': ['pctchg_30'],
                   'input': {'price': 'pctChg'},
                   'arg': {'timeperiod': 30}, 'batch': 30},
"pctchg_10":{'calc_cmd': 'ta', 'cmd': 'SUM', 'out': ['pctchg_10'],
                   'input': {'price': 'pctChg'},
                   'arg': {'timeperiod': 10}, 'batch': 10},
"pctchg_60":{'calc_cmd': 'ta', 'cmd': 'SUM', 'out': ['pctchg_60'],
                   'input': {'price': 'pctChg'},
                   'arg': {'timeperiod': 60}, 'batch': 60},
"pctchg_90":{'calc_cmd': 'ta', 'cmd': 'SUM', 'out': ['pctchg_90'],
                   'input': {'price': 'pctChg'},
                   'arg': {'timeperiod': 90}, 'batch': 90},
"pctchg_120":{'calc_cmd': 'ta', 'cmd': 'SUM', 'out': ['pctchg_120'],
                   'input': {'price': 'pctChg'},
                   'arg': {'timeperiod': 120}, 'batch': 120},
}
factors3={'tech_high_max60':{'calc_cmd': 'ta', 'cmd': 'MAX', 'out': ['tech_high_max60'],
                      'input': {'price': 'high'},
                      'arg': {'timeperiod': 60}, 'batch': 60},
'tech_high_max120':{'calc_cmd': 'ta', 'cmd': 'MAX', 'out': ['tech_high_max120'],
          'input': {'price': 'high'},
          'arg': {'timeperiod': 120}, 'batch': 120},
"pctchg_80":{'calc_cmd': 'ta', 'cmd': 'SUM', 'out': ['pctchg_80'],
               'input': {'price': 'pctChg'},
               'arg': {'timeperiod': 80}, 'batch': 80},
'tech_high':{'calc_cmd': 'ta', 'cmd': 'MEDPRICE', 'out': ['tech_high'],
                       'input': {'prices': ['high', 'high']}, 'arg': {}, 'batch': 1},
"ma_amount_20":{'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ma_amount_20'],
                   'input': {'price': 'amount'},
                    'arg': {'timeperiod': 20, 'matype': 0}, 'batch': 20},
"pctchg_20": {'calc_cmd': 'ta', 'cmd': 'SUM', 'out': ['pctchg_20'],
            'input': {'price': 'pctChg'},
            'arg': {'timeperiod': 20}, 'batch': 20},
"pctchg_50": {'calc_cmd': 'ta', 'cmd': 'SUM', 'out': ['pctchg_50'],
            'input': {'price': 'pctChg'},
            'arg': {'timeperiod': 50}, 'batch': 50},
"ma_close_150": {'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ma_close_150'],
                     'input': {'price': 'close'},
                     'arg': {'timeperiod': 150, 'matype': 0}, 'batch': 150},
'tech_close':{'calc_cmd': 'ta', 'cmd': 'MEDPRICE', 'out': ['tech_close'],
                       'input': {'prices': ['close', 'close']}, 'arg': {}, 'batch': 1},
          }
factors4={"pctchg_10": {'calc_cmd': 'ta', 'cmd': 'SUM', 'out': ['pctchg_10'],
                     'input': {'price': 'pctChg'},
                     'arg': {'timeperiod': 10}, 'batch': 10},
}
from factor.research.pl import nperiodpl as pl
factors2={
    "future_minmax_5":{'calc_cmd': 'function', 'function': pl.n_period_future_maxmin_byclose, 'arg': {"period":5,"pltype":"pcg","out":["min_5","max_5"]}},
"future_minmax_10":{'calc_cmd': 'function', 'function': pl.n_period_future_maxmin_byclose, 'arg': {"period":10,"pltype":"pcg","out":["min_10","max_10"]}},
"future_minmax_20":{'calc_cmd': 'function', 'function': pl.n_period_future_maxmin_byclose, 'arg': {"period":20,"pltype":"pcg","out":["min_20","max_20"]}},
"future_minmax_30":{'calc_cmd': 'function', 'function': pl.n_period_future_maxmin_byclose, 'arg': {"period":30,"pltype":"pcg","out":["min_30","max_30"]}},
"future_minmax_40":{'calc_cmd': 'function', 'function': pl.n_period_future_maxmin_byclose, 'arg': {"period":40,"pltype":"pcg","out":["min_40","max_40"]}},
"future_minmax_60":{'calc_cmd': 'function', 'function': pl.n_period_future_maxmin_byclose, 'arg': {"period":60,"pltype":"pcg","out":["min_60","max_60"]}},
"future_minmax_90":{'calc_cmd': 'function', 'function': pl.n_period_future_maxmin_byclose, 'arg': {"period":90,"pltype":"pcg","out":["min_90","max_90"]}},
"future_minmax_120":{'calc_cmd': 'function', 'function': pl.n_period_future_maxmin_byclose, 'arg': {"period":120,"pltype":"pcg","out":["min_120","max_120"]}},
}

factors21={
    "future_pl_5":{'calc_cmd': 'function', 'function': pl.n_period_hold_pl, 'arg': {"period":5,"pltype":"pcg","out":"future_pl_5"}},
"future_pl_10":{'calc_cmd': 'function', 'function': pl.n_period_hold_pl, 'arg': {"period":10,"pltype":"pcg","out":"future_pl_10"}},
"future_pl_20":{'calc_cmd': 'function', 'function': pl.n_period_hold_pl, 'arg': {"period":20,"pltype":"pcg","out":"future_pl_20"}},
"future_pl_30":{'calc_cmd': 'function', 'function': pl.n_period_hold_pl, 'arg': {"period":30,"pltype":"pcg","out":"future_pl_30"}},
"future_pl_40":{'calc_cmd': 'function', 'function': pl.n_period_hold_pl, 'arg': {"period":40,"pltype":"pcg","out":"future_pl_40"}},
"future_pl_60":{'calc_cmd': 'function', 'function': pl.n_period_hold_pl, 'arg': {"period":60,"pltype":"pcg","out":"future_pl_60"}},
"future_pl_90":{'calc_cmd': 'function', 'function': pl.n_period_hold_pl, 'arg': {"period":90,"pltype":"pcg","out":"future_pl_90"}},
"future_pl_120":{'calc_cmd': 'function', 'function': pl.n_period_hold_pl, 'arg': {"period":120,"pltype":"pcg","out":"future_pl_120"}},
}
factors30={
    "future_pl_2":{'calc_cmd': 'function', 'function': pl.n_period_hold_pl, 'arg': {"period":2,"pltype":"pcg","out":"future_pl_2"}},
"future_pl_3":{'calc_cmd': 'function', 'function': pl.n_period_hold_pl, 'arg': {"period":3,"pltype":"pcg","out":"future_pl_3"}},
"future_pl_4":{'calc_cmd': 'function', 'function': pl.n_period_hold_pl, 'arg': {"period":4,"pltype":"pcg","out":"future_pl_4"}},
"future_pl_50":{'calc_cmd': 'function', 'function': pl.n_period_hold_pl, 'arg': {"period":50,"pltype":"pcg","out":"future_pl_50"}},
"future_pl_140":{'calc_cmd': 'function', 'function': pl.n_period_hold_pl, 'arg': {"period":140,"pltype":"pcg","out":"future_pl_140"}},
"future_pl_150":{'calc_cmd': 'function', 'function': pl.n_period_hold_pl, 'arg': {"period":150,"pltype":"pcg","out":"future_pl_150"}},
"future_pl_180":{'calc_cmd': 'function', 'function': pl.n_period_hold_pl, 'arg': {"period":180,"pltype":"pcg","out":"future_pl_180"}},
"future_pl_200":{'calc_cmd': 'function', 'function': pl.n_period_hold_pl, 'arg': {"period":200,"pltype":"pcg","out":"future_pl_200"}},
    "future_minmax_2":{'calc_cmd': 'function', 'function': pl.n_period_future_maxmin_byclose, 'arg': {"period":2,"pltype":"pcg","out":["min_2","max_2"]}},
"future_minmax_3":{'calc_cmd': 'function', 'function': pl.n_period_future_maxmin_byclose, 'arg': {"period":3,"pltype":"pcg","out":["min_3","max_3"]}},
"future_minmax_4":{'calc_cmd': 'function', 'function': pl.n_period_future_maxmin_byclose, 'arg': {"period":4,"pltype":"pcg","out":["min_4","max_4"]}},
"future_minmax_50":{'calc_cmd': 'function', 'function': pl.n_period_future_maxmin_byclose, 'arg': {"period":50,"pltype":"pcg","out":["min_50","max_50"]}},
"future_minmax_140":{'calc_cmd': 'function', 'function': pl.n_period_future_maxmin_byclose, 'arg': {"period":140,"pltype":"pcg","out":["min_140","max_140"]}},
"future_minmax_150":{'calc_cmd': 'function', 'function': pl.n_period_future_maxmin_byclose, 'arg': {"period":150,"pltype":"pcg","out":["min_150","max_150"]}},
"future_minmax_180":{'calc_cmd': 'function', 'function': pl.n_period_future_maxmin_byclose, 'arg': {"period":180,"pltype":"pcg","out":["min_180","max_180"]}},
"future_minmax_200":{'calc_cmd': 'function', 'function': pl.n_period_future_maxmin_byclose, 'arg': {"period":200,"pltype":"pcg","out":["min_200","max_200"]}},

}
factors40={
'tech_high_max60':{'calc_cmd': 'ta', 'cmd': 'MAX', 'out': ['tech_high_max60'],
                      'input': {'price': 'high'},
                      'arg': {'timeperiod': 60}, 'batch': 60},
'tech_high_max120':{'calc_cmd': 'ta', 'cmd': 'MAX', 'out': ['tech_high_max120'],
          'input': {'price': 'high'},
          'arg': {'timeperiod': 120}, 'batch': 120},
'tech_high':{'calc_cmd': 'ta', 'cmd': 'MEDPRICE', 'out': ['tech_high'],
                       'input': {'prices': ['high', 'high']}, 'arg': {}, 'batch': 1}
}
#rps计算
def deal_rps(url=r"G:\factor\k1d",pctchg="pctchg_50",save_url="rps_50"):
    a=load(os.path.join(url,pctchg))
    rst={}
    for d,v in tqdm(a.items()):
        vd=pd.DataFrame([v],index=[pctchg]).T
        vd.dropna(inplace=True)
        vd1=vd
        #vd1=vd.loc[not np.isnan(vd[pctchg])]
        vd1.sort_values([pctchg],ascending=False,inplace=True)
        vd1["name"]=list(range(len(vd1), 0, -1))
        vd1["name"]=100*vd1["name"]/len(vd1)
        #vd1.index=vd1["obj"]
        rst[d]=vd1["name"].to_dict("index")
    save(rst,save_url)
    return rst
def cal_is_new_high(url=r"G:\factor\k1d",high_max="tech_high_max60",save_url="is_new_high",days = 5):
    a = load(os.path.join(url, high_max))
    a1 = load(os.path.join(url, 'tech_high'))
    rst = {}
    save_url = save_url + '_' + str(days)
    for d, v in tqdm(a.items()):
        #if (d <= datetime.datetime(2021, 8, 5)) & (d >= datetime.datetime(2000, 1, 1)):
        if (d >= datetime.datetime(2000, 1, 1)):
            f = {}
            for obj, v1 in a[d].items():
                if v1 <= a1[d][obj]:
                    if obj not in f.keys():
                        f[obj] = {}
                        f[obj]['is_new_high'] = 1
            rst[d] = f
    rst_fina = {}
    dates = list(rst.keys())
    for d, v in tqdm(rst.items()):
        f = {}
        start = dates[(dates.index(d)) - days+1]
        for d1 in get_calender_a(start=start, end=d):
            for obj, v1 in rst[d1].items():
                if v1['is_new_high'] >= 1:
                    if obj not in f.keys():
                        f[obj] = 1
                    else:
                        f[obj] = f[obj] + 1
        rst_fina[d] = f
    save(rst_fina,os.path.join(url, save_url))
    return rst_fina
#创新高天数
def deal_new_high_days(dataurl,save_url="new_high_days",args={}):
    rst={}
    for i in tqdm(os.listdir(dataurl)):
        obj = i.split("_")[0]
        k = load(os.path.join(dataurl, i))
        r=calc_new_high_days(k,args)
        for d,v in r.items():
            if d not in rst.keys():
                t={obj:v}
                rst[d]=t
            else:
                t=rst[d]
                t[obj]=v
    save(rst,save_url)
    return rst

def calc_new_high_days(k,args={}):
    """
    k:list
    """
    price=args.get("price","close")
    rst={}
    if len(k)==1:
        for i in range(len(k)):
            t=k[i]
            date=t["timekey"]
            k1=k[:i]
            n=0
            rst[date]=n
    else:
        for i in range(len(k)):
            t=k[i]
            date=t["timekey"]
            k1=k[:i]
            n=0
            for tt in reversed(k1):
                if t[price]>tt[price]:
                    n=n+1
                else:
                    break
            rst[date]=n
    return rst
#创新低天数
def deal_new_low_days(dataurl,save_url="new_low_days",args={}):
    rst={}
    for i in tqdm(os.listdir(dataurl)):
        obj = i.split("_")[0]
        k = load(os.path.join(dataurl, i))
        r=calc_new_low_days(k,args)
        for d,v in r.items():
            if d not in rst.keys():
                t={obj:v}
                rst[d]=t
            else:
                t=rst[d]
                t[obj]=v
    save(rst, save_url)
    return rst
def calc_new_low_days(k,args={}):
    """
    k:list
    """
    price=args.get("price","close")
    rst={}
    if len(k)==1:
        for i in range(len(k)):
            t=k[i]
            date=t["timekey"]
            k1=k[:i]
            n=0
            rst[date]=n
    else:
        for i in range(len(k)):
            t=k[i]
            date=t["timekey"]
            k1=k[:i]
            n=0
            for tt in reversed(k1):
                if t[price]<tt[price]:
                    n=n+1
                else:
                    break
            rst[date]=n
    return rst


#N天内创新高新低数量
class ndays_appear:
    def getargsout(self,arg):
        if "out" in arg.keys():
            return arg["out"]
        elif "arg" in arg.keys():
            if "out" in arg["arg"].keys():
                return arg["arg"]["out"]
            else:
                print("getargsout err")
                return []
        else:
            print("getargsout err")
            return []
    def calc(self,factor_data,args,key=None):
        data={}
        for date,d in factor_data.items():
            for obj,v in d.items():
                if obj not in data.keys():
                    data[obj]=[]
                t={"timekey":date}
                if key:
                    t["data"]=v[key]
                else:
                    t["data"] = v
                data[obj].append(t)
        rst={}
        for obj,d in data.items():
            kd=pd.DataFrame(d)
            kd.index=kd["timekey"]
            if isinstance(args, list):
                kd = graph.run(kd, args)
                out = self.getargsout(args[-1])
            else:
                out = self.getargsout(args)
                kd = graph.run(kd, [args])
            if len(out)>1:
                for k1, v1 in kd[out].to_dict("index").items():
                    if isinstance(k1, pd.Timestamp):
                        k1 = k1.to_pydatetime()
                    if k1 not in rst.keys():
                        rst[k1]={}
                    rst[k1][obj] = v1
            else:
                o=out[0]
                for k1, v1 in kd[o].to_dict("index").items():
                    if isinstance(k1, pd.Timestamp):
                        k1 = k1.to_pydatetime()
                    if k1 not in rst.keys():
                        rst[k1]={}
                    rst[k1][obj] = v1
        return rst
    def calc_save(self,source_factor,to_factor,args,key=None):
        f=load(source_factor)
        a=self.calc(f,args,key=key)
        save(a,to_factor)

def deal_daily(ip="127.0.0.1",location=r"G:/factor/k1d/new1"):
    db = getdb_client("stock", ip)
    trade_info={}
    finance_info={}
    size_info={}
    db = getdb_client("stock", ip)
    objs_delist = []
    for i in db["stock_info"].find({}):
        i["start"]=str2datetime(i["list_date"])
        if i["delist_date"]:
            i["end"] = str2datetime(i["delist_date"])
        else:
            i["end"]=datetime.datetime(2021,8,6)
        objs_delist.append(i)
    for i in tqdm(db["stock_daily_basic"].find({})):
        d=i["trade_date"]
        d=datetime.datetime.strptime(d,"%Y%m%d")
        obj=i["obj"]
        t0 = {"close": i["close"], "turnover_rate": i["turnover_rate"], "turnover_rate_f": i["turnover_rate_f"],
             "volume_ratio": i["volume_ratio"]}
        t1 = {"pe": i["pe"], "pe_ttm": i["pe_ttm"], "pb": i["pb"], "ps": i["ps"], "ps_ttm": i["ps_ttm"],
              "dv_ratio": i["dv_ratio"], "dv_ttm": i["dv_ttm"]}

        t2 = {"total_share": i["total_share"], "float_share": i["float_share"], "free_share": i["free_share"],
              "total_mv": i["total_mv"], "circ_mv": i["circ_mv"]}
        if d not in trade_info.keys():
            trade_info[d]={}
            trade_info[d][obj]=t0
        else:
            t=trade_info[d]
            t[obj]=t0
        if d not in finance_info.keys():
            finance_info[d]={}
            finance_info[d][obj]=t1
        else:
            t=finance_info[d]
            t[obj]=t1
        if d not in size_info.keys():
            size_info[d]={}
            size_info[d][obj]=t2
        else:
            t=size_info[d]
            t[obj]=t2
    t1={}
    t2={}
    t3={}
    ds=list(trade_info.keys())
    ds.sort()
    for d in tqdm(get_calender_a(ds[0], ds[-1])):
        if d not in trade_info.keys():
            trade_info[d]=deepcopy(t1)
        else:
            t=trade_info[d]
            for i in objs_delist:
                obj=i["obj"]
                if d >=i["start"] and d<=i["end"]:
                    if obj not in t.keys():
                        if obj  in t1.keys():
                            t[obj]=deepcopy(t1[obj])
                        else:
                            print(obj,d)
                    else:
                        t1[obj]=t[obj]
        if d not in finance_info.keys():
            finance_info[d]=deepcopy(t2)
        else:
            t=finance_info[d]
            for i in objs_delist:
                obj=i["obj"]
                if d >=i["start"] and d<=i["end"]:
                    if obj not in t.keys():
                        if obj  in t2.keys():
                            t[obj]=deepcopy(t2[obj])
                        else:
                            print(obj,d)
                    else:
                        t2[obj]=t[obj]
        if d not in size_info.keys():
            size_info[d]=deepcopy(t3)
        else:
            t=size_info[d]
            for i in objs_delist:
                obj=i["obj"]
                if d >=i["start"] and d<=i["end"]:
                    if obj not in t.keys():
                        if obj  in t3.keys():
                            t[obj]=deepcopy(t3[obj])
                        else:
                            print(obj,d)
                    else:
                        t3[obj]=t[obj]
    save(trade_info,os.path.join(location,"trade_info"))
    save(finance_info, os.path.join(location, "finance_info"))
    save(size_info, os.path.join(location, "size_info"))
    return trade_info,finance_info,size_info
def deal_bypool0(ispool=True,location=r"G:/factor/k1d/new1",refresh=True):
    from factor.research.factor import factor_ta
    url = r"G:\data\stock\hfq\k1d\byobj"
    batch_args=[
        (deal_rps,(location, "pctchg_50", os.path.join(location,"rps_50"),),os.path.join(location,"rps_50.pkl")),
        (deal_rps,(location, "pctchg_10", os.path.join(location, "rps_10"),),os.path.join(location,"rps_10.pkl")),
        (deal_rps, (location, "pctchg_20", os.path.join(location, "rps_20"),),os.path.join(location,"rps_20.pkl")),
        (deal_rps, (location, "pctchg_40", os.path.join(location, "rps_40"),),os.path.join(location,"rps_40.pkl")),
        (deal_rps, (location, "pctchg_60", os.path.join(location, "rps_60"),),os.path.join(location,"rps_60.pkl")),
        (deal_rps, (location, "pctchg_90", os.path.join(location, "rps_90"),),os.path.join(location,"rps_90.pkl")),
        (deal_rps, (location, "pctchg_120", os.path.join(location, "rps_120"),),os.path.join(location,"rps_120.pkl")),
        (deal_new_high_days,(url, os.path.join(location, "new_high_days"), {},),os.path.join(location,"new_high_days.pkl")),
        (deal_new_high_days, (url, os.path.join(location, "new_high_days_byhigh"), {"price":"high"},),os.path.join(location,"new_high_days_byhigh.pkl")),
        (deal_new_low_days,(url, os.path.join(location, "new_low_days"), {},),os.path.join(location,"new_low_days.pkl")),
        (deal_new_low_days,(url, os.path.join(location, "new_low_days_bylow"), {"price":"low"},),os.path.join(location,"new_low_days_bylow.pkl"))
    ]
    if ispool:
        import multiprocessing
        n = multiprocessing.cpu_count() - 2
        if n < len(batch_args):
            n = len(batch_args)
        pool = multiprocessing.Pool(n)
        for i in batch_args:
            if not refresh and os.path.exists(i[2]):
                print("存在", i[2])
            else:
                pool.apply_async(i[0], i[1])
                print(i)

        pool.close()
        pool.join()
    else:
        pass
def deal_redata_bypool(ispool=True,location=r"G:/factor/k1d/new1",refresh=True):
    from factor.research.factor import factor_ta
    url = r"G:\data\stock\hfq\k1d\byobj"
    batch_args = {}
    # batch_args.update(factors)
    # batch_args.update(factors1)
    # batch_args.update(factors21)
    batch_args.update(factors30)
    f = factor_ta(dataurl=url,iscutpctCh=False)
    if ispool:
        import multiprocessing
        n=multiprocessing.cpu_count() - 2
        if n <len(batch_args):
            n=len(batch_args)
        pool = multiprocessing.Pool(n)
        for name, args in batch_args.items():
            pool.apply_async(f.redata, (name, location,))
        pool.close()
        pool.join()
    else:
        for name, args in batch_args.items():
            f.redata(name, location)
def deal_bypool(ispool=True,location=r"G:/factor/k1d/new1",refresh=True):
    from factor.research.factor import factor_ta
    url = r"G:\data\stock\hfq\k1d\byobj"
    batch_args={}
    #batch_args.update(factors)
    #batch_args.update(factors1)
    # batch_args.update(factors21)
    batch_args.update(factors4)
    f = factor_ta(dataurl=url,iscutpctCh=False)
    f.calc_history_batch(location=location,k1d_url=url,batch_args=batch_args,refresh=refresh,ispool=ispool)
    print(datetime.datetime.now())
    deal_redata_bypool(ispool=ispool,location=location,refresh=refresh)
if __name__ == '__main__':
    import datetime
    print(datetime.datetime.now())
    deal_bypool(True,location=r"G:/factor/k1d/new1",refresh=True)
    #deal_bypool0()
    #deal_daily()
    print(datetime.datetime.now())
